# -*- coding:utf-8 -*-
'''
Created on 2012-7-17

@author: duoyi
'''
import datetime

from http.request import Request
import funcs
import settings

class Scheduler(object):
    def __init__(self,d_locks):
        self.d_locks=d_locks

        self.get_request_times=0        
        self.cxn=funcs.get_db_cxn()
        self.cur=self.cxn.cursor()
        self.start()
        
    def relogin(self):
        '''登录网站'''
        funcs.login_website()
        
    def start(self):
        '''启动，建立内存块'''
        self.relogin()
        funcs.create_tables(self.cur,self.cxn,settings.S_target_website)            #创建数据库表
        self.build_url_set()
        #判断是否更新
        if not settings.S_is_update:
            sql='select count(*) from %s_queue'%settings.S_target_website
            self.cur.execute(sql)
            count=self.cur.fetchone()[0]
            if count<=0:
                self.add_request(Request(settings.S_start_urls[0]),self.cur,self.cxn)
            else:
                pass
        else:
            sql='create table %s_update select (id) from %s order by id'%(settings.S_target_website,settings.S_target_website)
            try:
                self.cur.execute(sql)               #尝试建更新表
                sql='Alter table %s_update add primary key(id)'%settings.S_target_website           #增加主键
                self.cur.execute(sql)
            except Exception:                       #建表失败，表明更新表已存在
                sql='select count(*) from %s_update'%settings.S_target_website
                self.cur.execute(sql)
                count=self.cur.fetchone()[0]
                #若更新列表为空，则重新建表
                if count<=0:
                    sql='drop table %s_update'%settings.S_target_website
                    self.cur.execute(sql) 
                    #建表
                    sql='create table %s_update select (id) from %s order by id'%(settings.S_target_website,settings.S_target_website)  #增加主键
                    self.cur.execute(sql)
                    sql='Alter table %s_update add primary key(id)'%settings.S_target_website
                    self.cur.execute(sql)
        self.cxn.commit()
    def get_table_data(self,table_name):
        '''读取数据库表的数据'''
        print '    reading %s table'%table_name
        sql='select count(*) from %s'%table_name
        self.cur.execute(sql)
        count=self.cur.fetchone()[0]
        record_num=0
        if count>0:
            sql='select min(id),max(id) from %s'%table_name
            self.cur.execute(sql)
            min_id,max_id=self.cur.fetchone()            
            print '    ',min_id,max_id
            step=10**6
            idx=min_id
            while idx<=max_id:
                sql_pref='select id,url from %s'%table_name
                sql=sql_pref+' where id between %s and %s'
                self.cur.execute(sql,(str(idx),str(idx+step)))
                for record in self.cur.fetchall():
                    if 'queue' in table_name:
                        #url_queue的md5值需要重新计算
                        url=record[1]
                        i_md5=funcs.get_md52int(url)
                    else:
                        _id=record[0]
                        i_md5=int(_id)
                    if self.set_url_mark(i_md5):
                        record_num+=1
                idx+=step
        print '    done,%s has %d records'%(table_name,record_num)
        
    def build_url_set(self):
        '''建立内存块'''
        print 'building url_set'
        #前13位为段地址，11位为段偏移地址，最后5位为整型地址
        self.url_set=[0]*(2**funcs.SEG)
        for i in xrange(len(self.url_set)):
            self.url_set[i]=[0]*(2**funcs.SEG_OFFSET)
        
        for table in settings.S_website_tables:
            self.get_table_data(table)
        
        print 'url_set building success'
        #self.check_url_mark_num()
    def check_url_mark_num(self):
        '''测试内存块'''
        mark_num=0
        for seg in self.url_set:
            for duan in seg:
                for i in range(32):
                    bbit=(duan>>i)%2
                    if bbit==1:
                        mark_num+=1
        print 'has %d records'%mark_num
    
    def set_url_mark(self,l_md5):
        '''设置内存块的位'''
        self.d_locks['set_url_mark_lock'].acquire()
        seg_index,seg_offset,offset_value=funcs.get_bit_size(l_md5)
        u_offset_value=self.url_set[seg_index][seg_offset]
        b_u_o_b=(u_offset_value>>offset_value)%2
        if b_u_o_b==0:
            u_offset_value=u_offset_value|(1<<offset_value)
            self.url_set[seg_index][seg_offset]=u_offset_value
            return_value=True
        else:
            return_value=False
        self.d_locks['set_url_mark_lock'].release()
        return return_value
    
    def get_request(self,cur,cxn):
        '''从数据库不同表获取网址'''
        self.d_locks['get_request_lock'].acquire()
        if not settings.S_is_update:
            sql='select * from %s_queue where id=(select min(id) from %s_queue)'%(settings.S_target_website,settings.S_target_website)
            cur.execute(sql)
            record=cur.fetchone()
            if record:
                idx=record[0]
                url=record[1]
                d_sql_prefix='delete from %s_queue'%settings.S_target_website
                d_sql=d_sql_prefix+' where id=%s'
                cur.execute(d_sql,str(idx))
                cxn.commit()
                return_value=Request(url)
            else:
                return_value=None
        else:
            inner_sql='(select min(id) from %s_update)'%settings.S_target_website
            sql_prefix='select * from %s '%settings.S_target_website
            sql=sql_prefix+'where id='+inner_sql
            cur.execute(sql)
            record=cur.fetchone()
            if record:
                idx=record[0]
                url=record[1]
                sql_prefix='delete from %s_update '%settings.S_target_website
                d_sql=sql_prefix+'where id=%s'
                cur.execute(d_sql,str(idx))
                cxn.commit()
                
                update_fre=record[4]
                if update_fre:
                    update_fre=int(update_fre)
                    last_update_time=record[2]
                    time_format='%Y-%m-%d %H:%M:%S'
                    ldt=datetime.datetime.strptime(last_update_time,time_format)
                    now=datetime.datetime.now()
                    if (now-ldt).days>=update_fre:
                        #更新
                        return_value=Request(url)
                    else:
                        return_value="%s It's not the time to update."%url                        
                else:
                    #初始化
                    sql_prefix='update %s '%settings.S_target_website
                    sql=sql_prefix+'SET update_fre =%s WHERE id =%s'
                    cur.execute(sql,(str(1),str(idx)))
                    cxn.commit()
                    return_value=Request(url)
            else:
                return_value=None
        self.get_request_times+=1
        if self.get_request_times%2000==1:
            self.relogin()
        self.d_locks['get_request_lock'].release()
        return return_value
    
    def add_request(self,request,cur,cxn):
        '''添加网址进数据库表'''
        self.d_locks['add_request_lock'].acquire()
        l_md5=funcs.get_md52int(request.url)
        if self.set_url_mark(l_md5):
            sql_prefix='insert into %s_queue'%settings.S_target_website
            sql=sql_prefix+' (url) values(%s)'
            cur.execute(sql,request.url)
            cxn.commit()
        else:
            pass
        self.d_locks['add_request_lock'].release()
        
if __name__=='__main__':
    urls=['http://www.baidu.com','http://www.baidu.com','http://www.bfaidu.com/','http://www.baidu.com3/','http://www.baidu.com4/','http://www.baidu.com/']
    sche=Scheduler()
    for url in urls:
        r=Request(url)
        sche.add_request(r)
    while True:
        pass



    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    